register '$PIGGYBANK_PATH';

DEFINE DATE_EXTRACT_MM 
org.apache.pig.piggybank.evaluation.util.apachelogparser.DateExtractor('yyyy-MM');

DEFINE DATE_EXTRACT_DD 
org.apache.pig.piggybank.evaluation.util.apachelogparser.DateExtractor('yyyy-MM-dd');

-- pig/input/test-pig-full.txt
raw_logs = load '$INPUT' USING org.apache.pig.piggybank.storage.MyRegExLoader('^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] "(\\S+) (\\S+) (HTTP[^"]+)" (\\S+) (\\S+) "([^"]*)" "([^"]*)" "(\\S+)" "(\\S+)" (\\S+) "(.*)" (\\S+) (\\S+)')
as (remoteAddr: chararray, 
n2: chararray, 
n3: chararray, 
time: chararray, 
method: chararray,
path:chararray,
protocol:chararray,
status: int, 
bytes_string: chararray, 
referrer: chararray, 
browser: chararray, 
n10:chararray,
remoteLogname: chararray, 
remoteAddr12: chararray, 
path2: chararray, 
sessionid: chararray, 
n15: chararray
);

filter_logs = FILTER raw_logs BY not (browser matches '.*pingdom.*');
--item_logs = FOREACH raw_logs GENERATE browser;

--status
all_status_logs = FOREACH filter_logs  GENERATE status;
distinct_status_logs = DISTINCT all_status_logs;
STORE distinct_status_logs into '$OUTPUT' using PigStorage(',');

/*
--user
all_users_logs = FOREACH filter_logs  GENERATE sessionid;
distinct_users_logs = DISTINCT all_users_logs;
STORE distinct_users_logs into '$OUTPUT' using PigStorage(',');


all_date_logs = FOREACH filter_logs  GENERATE DATE_EXTRACT_MM(time) as month,DATE_EXTRACT_DD(time) as day;
--distinct_date = DISTINCT all_date_logs;
STORE all_date_logs into '$OUTPUT' using PigStorage(',');


--unique 404 paths
filter_unique_paths_logs = FILTER filter_logs by status == 404;
reitem_unique_paths_logs = FOREACH filter_unique_paths_logs GENERATE status,path;
distinct_unique_paths_logs = DISTINCT reitem_unique_paths_logs;
DUMP distinct_unique_paths_logs
STORE reitem_unique_paths_logs INTO '$OUTPUT' using PigStorage(',');


--month 500 404 logs
filter_month_500_404_logs = FILTER filter_logs BY status == 500 or status ==404;
reitem_month_500_404_logs = FOREACH filter_month_500_404_logs GENERATE status, DATE_EXTRACT_MM(time) as month;
group_month_500_404_logs = GROUP reitem_month_500_404_logs BY (status,month);
final_month_500_404_logs = FOREACH group_month_500_404_logs GENERATE group ,COUNT($1);
STORE final_month_500_404_logs into '$OUTPUT' using PigStorage(',');


--day 500 404 logs
filter_day_500_404_logs = FILTER filter_logs BY status == 500 or status ==404;
reitem_day_500_404_logs = FOREACH filter_day_500_404_logs GENERATE status, DATE_EXTRACT_DD(time) as day;
group_day_500_404_logs = GROUP reitem_day_500_404_logs BY (status,day);
final_day_500_404_logs = FOREACH group_day_500_404_logs GENERATE group ,COUNT($1);
STORE final_day_500_404_logs into '$OUTPUT' using PigStorage(',');
*/
/*


--percent 500 logs
reitem_percent_500_logs = FOREACH filter_logs GENERATE status,DATE_EXTRACT_MM(time) as month;
group_month_percent_500_logs = GROUP reitem_percent_500_logs BY (month);
final_month_500_logs = FOREACH group_month_percent_500_logs 
{
	total = COUNT(reitem_percent_500_logs);
	t = filter reitem_percent_500_logs by status== 500; --create a bag which contains only T values
	generate flatten(group) as col1, total, COUNT(t), 100*(double)COUNT(t)/(double)total;
}
STORE final_month_500_logs into '$OUTPUT' using PigStorage(',');
*/

--percent 404 logs
reitem_percent_404_logs = FOREACH filter_logs GENERATE status,DATE_EXTRACT_MM(time) as month;
group_month_percent_404_logs = GROUP reitem_percent_404_logs BY (month);
final_month_404_logs = FOREACH group_month_percent_404_logs 
{
	total = COUNT(reitem_percent_404_logs);
	t = filter reitem_percent_404_logs by status== 404; --create a bag which contains only T values
	generate flatten(group) as col1, total, COUNT(t), 100*(double)COUNT(t)/(double)total;
}
STORE final_month_404_logs into '$OUTPUT' using PigStorage(',');


--logs_base = FOREACH raw_logs GENERATE FLATTEN(REGEX_EXTRACT_ALL('192.168.1.5:8020', '(.*)\\:(.*)'));
--dump raw_logs;






